• Purposes of text cleaning

• Getting rid of noise and stop words

• Tokenizing and Ngrams

• Stemming and lemmatizing

• Language guesser

• POS tagging

library (cld2)         # for language detection
library (corpus)       # for text analysis with support for international text
library (dplyr)         # for data manipulation
library (entity)        # for easy NER 
library (ggraph)       # for graphs
library (gridExtra)     # for working with grids to obtain nice layouts
library (hunspell)     # for high-performance stemming
library (igraph)       # for easy graphs 
library (janitor)      # for a pretty table
library (lattice)      # for easy charts
library (knitr)        # for manipulating the output of this markdown document
library (plyr)         # for data manipulation
library (NLP)          # for natural language processing tools
library (openNLP)      # for named entity recognition
library (pander)       # for nice slide output
library (qdap)         # for using parsing tools to prepare transcript data

setwd("C:/Noodfolder")

• Articles on John Maynard Keynes taken from The Times newspaper

• read.csv (“Keynes.csv”)

Keynes <- read.csv("Keynes.csv", stringsAsFactors = FALSE)

• Title: Mr. Roosevelt’s Experiments
• Publication date: February 27, 1940

TEXT <- "HOW TO PAY FOR THE 3WAR. By JOHN\nMAYNARD KEYNES. Macmillan. Is.
\nIn three articles published i6n Tlhe Times last November Mr. J. M.
Keynes put forward what was expressly a first draft of proposals f6or
compulsory savings in  wartime. 3388 His plan  now reappears in the form
of  a pamphlet in which Mr. Keynes elaborates his earlier argument and
varies it to meet the  comment and suggestion which it has so widely
provoked.The pamphlet includes four appendices on the national income5the 
extent of our resources abroad the cost of family allowances and\nthe formula 
for the aggregate of deferred pay and direct taxes\nthe\npamphlet  
is the subject of a leading article in 344 todays issue"

TEXTtokens <- str_split(TEXT, " ")
TEXTtokens <- unlist(TEXTtokens)
print (TEXTtokens [1:50])

##  [1] "HOW"              "TO"               "PAY"              "FOR"             
##  [5] "THE"              "3WAR."            "By"               "JOHN\nMAYNARD"   
##  [9] "KEYNES."          "Macmillan."       "Is.\n\nIn"        "three"           
## [13] "articles"         "published"        "i6n"              "Tlhe"            
## [17] "Times"            "last"             "November"         "Mr."             
## [21] "J."               "M.\nKeynes"       "put"              "forward"         
## [25] "what"             "was"              "expressly"        "a"               
## [29] "first"            "draft"            "of"               "proposals"       
## [33] "f6or\ncompulsory" "savings"          "in"               ""                
## [37] "wartime."         "3388"             "His"              "plan"            
## [41] ""                 "now"              "reappears"        "in"              
## [45] "the"              "form\nof"         ""                 "a"               
## [49] "pamphlet"         "in"

wordcloud(TEXTtokens, scale = c(4,.5), min.freq =1, colors = brewer.pal(1, "Dark2"))

TEXT2 <- tolower(TEXT)
cat(TEXT2)

## how to pay for the 3war. by john
## maynard keynes. macmillan. is.
## 
## in three articles published i6n tlhe times last november mr. j. m.
## keynes put forward what was expressly a first draft of proposals f6or
## compulsory savings in  wartime. 3388 his plan  now reappears in the form
## of  a pamphlet in which mr. keynes elaborates his earlier argument and
## varies it to meet the  comment and suggestion which it has so widely
## provoked.the pamphlet includes four appendices on the national income5the 
## extent of our resources abroad the cost of family allowances and
## the formula 
## for the aggregate of deferred pay and direct taxes
## the
## pamphlet  
## is the subject of a leading article in 344 todays issue

TEXT2 <- tm::removeNumbers(TEXT2)
cat(TEXT2)

## how to pay for the war. by john
## maynard keynes. macmillan. is.
## 
## in three articles published in tlhe times last november mr. j. m.
## keynes put forward what was expressly a first draft of proposals for
## compulsory savings in  wartime.  his plan  now reappears in the form
## of  a pamphlet in which mr. keynes elaborates his earlier argument and
## varies it to meet the  comment and suggestion which it has so widely
## provoked.the pamphlet includes four appendices on the national incomethe 
## extent of our resources abroad the cost of family allowances and
## the formula 
## for the aggregate of deferred pay and direct taxes
## the
## pamphlet  
## is the subject of a leading article in  todays issue

TEXT2<- str_replace_all(TEXT2, "  ", " ")
cat(TEXT2)

## how to pay for the war. by john
## maynard keynes. macmillan. is.
## 
## in three articles published in tlhe times last november mr. j. m.
## keynes put forward what was expressly a first draft of proposals for
## compulsory savings in wartime. his plan now reappears in the form
## of a pamphlet in which mr. keynes elaborates his earlier argument and
## varies it to meet the comment and suggestion which it has so widely
## provoked.the pamphlet includes four appendices on the national incomethe 
## extent of our resources abroad the cost of family allowances and
## the formula 
## for the aggregate of deferred pay and direct taxes
## the
## pamphlet 
## is the subject of a leading article in todays issue

TEXT2<-tm::removePunctuation(TEXT2)
cat(TEXT2)

## how to pay for the war by john
## maynard keynes macmillan is
## 
## in three articles published in tlhe times last november mr j m
## keynes put forward what was expressly a first draft of proposals for
## compulsory savings in wartime his plan now reappears in the form
## of a pamphlet in which mr keynes elaborates his earlier argument and
## varies it to meet the comment and suggestion which it has so widely
## provokedthe pamphlet includes four appendices on the national incomethe 
## extent of our resources abroad the cost of family allowances and
## the formula 
## for the aggregate of deferred pay and direct taxes
## the
## pamphlet 
## is the subject of a leading article in todays issue

testtokens <- text_ntoken(TEXT)
print(testtokens)

## [1] 126

testtokens2 <- text_ntoken(TEXT2)
print(testtokens2)

## [1] 115

#All words in lower case  
cleancontent <- tolower(Keynes$content)

## Remove numbers  

cleancontent <- tm::removeNumbers(cleancontent)

## Remove double spaces  

cleancontent<- str_replace_all(cleancontent, "  ", " ")

## Remove punctuation

cleancontent<-tm::removePunctuation(cleancontent)

## Store results in new column in the dataframe

Keynes$clean_content <- cleancontent

testtokens <- text_ntoken(Keynes$content)
print (sum(testtokens))

## [1] 141015

testtokens2 <- text_ntoken(Keynes$clean_content)
print (sum(testtokens2))

## [1] 116667

#Stopwordslist  
  
Joseesstopwords <- c("the", "a", "an", "and", "i","in", "ïf", "he","she",
                     "him", "should", "or", "we", "no", "over", "only",
                     "could", "to", "of", "it", "is", "that", "this",
                     "was", "/'s", "with","/'t", "as", "'t", "on", "are",
                     "one", "so", "be","me", "are", "at", "s", "has",
                     "by", "/", "for", "not", "from", "have", "which",
                     "but", "his","than", "their", "were", "any", "some",
                     "can", "what", "will", "would", "been", "per",
                     "more", "they", "there", "tibble", "its", "had",
                     "mrs", "mr",   "when", "all", "our", "cent", "her",
                     "who") 

Keynestext <- removeWords(Keynes$clean_content, Joseesstopwords)
#And store clean text in a new field
Keynes$clean_content <- Keynestext

SnowballC library uses Porter’s word stemming algorithm that collapses words to a common root. It supports many languages:

getStemLanguages()

##  [1] "arabic"     "basque"     "catalan"    "danish"     "dutch"     
##  [6] "english"    "finnish"    "french"     "german"     "greek"     
## [11] "hindi"      "hungarian"  "indonesian" "irish"      "italian"   
## [16] "lithuanian" "nepali"     "norwegian"  "porter"     "portuguese"
## [21] "romanian"   "russian"    "spanish"    "swedish"    "tamil"     
## [26] "turkish"

wordStem("compulsary")

## [1] "compulsari"

wordStem("articles")

## [1] "articl"

wordStem("published")

## [1] "publish"

wordStem("November")

## [1] "Novemb"

## [1] "how to pay for the war by john\nmaynard keynes macmillan is\n\nin three articles published in tlhe times last november mr j m\nkeynes put forward what was expressly a first draft of proposals for\ncompulsory savings in wartime his plan now reappears in the form\nof a pamphlet in which mr keynes elaborates his earlier argument and\nvaries it to meet the comment and suggestion which it has so widely\nprovokedthe pamphlet includes four appendices on the national incomethe \nextent of our resources abroad the cost of family allowances and\nthe formula \nfor the aggregate of deferred pay and direct taxes\nthe\npamphlet \nis the subject of a leading article in todays issu"

how to pay for the war by john maynard keynes macmillan Is in three articles published in the times last november mr j m keynes put forward what was expressly a first draft of proposals for compulsory savings in wartime

how to pay for the war by john maynard keyn macmillan is in three articl publish in the time last novemb mr j m keyn put forward what was expressli a first draft of propos for compulsori save in wartim

https://github.com/michmech/lemmatization-lists

make_lemma_dictionary(TEXT2)

##         token     lemma
## 1          is         i
## 2    articles   article
## 3   published   publish
## 4       times      time
## 5   expressly   express
## 6   proposals  proposal
## 7     savings      save
## 8         his        hi
## 9   reappears    appear
## 10 elaborates elaborate
## 11    earlier     early
## 12     varies      vary
## 13    comment       com
## 14        has        ha
## 15     widely      wide
## 16   includes   include
## 17  resources    source
## 18 allowances allowance
## 19      taxes       tax
## 20    leading      lead

Lemmatext <- lemmatize_strings(TEXT2, dictionary = lexicon::hash_lemmas)

• Package cld = Compact Language Detector (Google)
• several versions (2, 3)

GRreviewssc <- read.csv("CLDdemo2.csv", sep = ";")
GRreviewssc$langdtcd <- detect_language(GRreviewssc$Text, 
plain_text = TRUE, lang_code = TRUE)

## how to pay for the war by john
## maynard keynes macmillan is
## 
## in three articles published in tlhe times last november mr j m
## keynes put forward what was expressly a first draft of proposals for
## compulsory savings in wartime his plan now reappears in the form
## of a pamphlet in which mr keynes elaborates his earlier argument and
## varies it to meet the comment and suggestion which it has so widely
## provokedthe pamphlet includes four appendices on the national incomethe 
## extent of our resources abroad the cost of family allowances and
## the formula 
## for the aggregate of deferred pay and direct taxes
## the
## pamphlet 
## is the subject of a leading article in todays issue

ud_model <- udpipe_download_model(language = "english")

ud_english <- udpipe_load_model(ud_model$file_model)
cleanfieldPOS <- udpipe_annotate(ud_english, x = Keynes$clean_content, doc_id = Keynes$date.pub)
cleanfieldPOS <- as.data.frame(cleanfieldPOS)

statsclean <- txt_freq(cleanfieldPOS$upos)
statsclean$key <- factor(statsclean$key, levels = rev(statsclean$key))
barchart(key ~ freq, data = statsclean, col = "cadetblue", 
main = "UPOS (Universal Parts of Speech)\n frequency of occurrence", 
 xlab = "Frequency")

nounstat <- subset(cleanfieldPOS, upos %in% c("NOUN")) 
nounstat <- txt_freq(nounstat$token)

keyrake <- keywords_rake(x = cleanfieldPOS, term = "lemma", group = "doc_id", 
                       relevant = cleanfieldPOS$upos %in% c("NOUN", "ADJ"))

Correct NER tagging requires a model for the language used. Wirtschafts universität in Vienna offers models on this website: https://datacube.wu.ac.at/.

President <- readLines('Debat.txt')
President <- as.String(President)

Open NLP needs the file to be tokenized, either in words or in sentences and the result will be stored as a list.

sent_ann <- Maxent_Sent_Token_Annotator()

President_sent <- NLP::annotate(President, list(sent_ann))
President_nmbrd <- President[President_sent]
President_nmbrd

##  [1] "President <- \"Entering into the debate, Biden had a significant and persistent lead in the polls."                                                                                                                                                                                                                                                                                           
##  [2] "Biden's lead was compounded by a funding shortage in Trump's campaign, with Biden's campaign donations improving significantly."                                                                                                                                                                                                                                                              
##  [3] "Since Biden's successful nomination in the Democratic primaries, Trump had attempted to cast doubt over Biden's abilities, claiming that he was suffering from dementia and that he was taking performance-enhancing drugs in the primaries."                                                                                                                                                 
##  [4] "Trump called for Biden to be drug tested before the debate."                                                                                                                                                                                                                                                                                                                                  
##  [5] "Biden mocked the idea.[20]"                                                                                                                                                                                                                                                                                                                                                                   
##  [6] "Trump also claimed that Biden would use a hidden electronic earpiece for the debate, demanding that Biden's ears be searched."                                                                                                                                                                                                                                                                
##  [7] "Again, Biden declined."                                                                                                                                                                                                                                                                                                                                                                       
##  [8] "Running up to the debate, Trump made repeated claims that the election would be rigged by means of voter fraud, especially with regards to mail-in ballots."                                                                                                                                                                                                                                  
##  [9] "When asked if he would commit to a peaceful transition of power, Trump said, we'll have to wait and see; however, in a later press briefing, he said that he did believe in a peaceful transition of power.In several instances, Trump called for his supporters to vote twice—in order to test safeguards against voter fraud—even though voting more than once is a felony."                
## [10] "In the weeks leading up to the debate, Trump became part of various controversies."                                                                                                                                                                                                                                                                                                           
## [11] "Bob Woodward released his second book on the Trump presidency, based on 19 recorded interviews with Trump."                                                                                                                                                                                                                                                                                   
## [12] "In one recording made in February 2020, Trump indicated that he understood the severity of the COVID-19 pandemic early on, which contrasted with Trump's attempts to publicly play down the virus."                                                                                                                                                                                           
## [13] "Trump confirmed that he downplayed the severity of the pandemic, saying that I don't want to create a panic."                                                                                                                                                                                                                                                                                 
## [14] "The New York Times published an investigation into Trump's federal tax returns, which found that Trump had paid no tax at all in 10 out of 15 years studied, and only $750 in federal income tax for 2016 and 2017."                                                                                                                                                                          
## [15] "Additionally, they reported that his businesses lost money in most years."                                                                                                                                                                                                                                                                                                                    
## [16] "A few days before the debate, the US reached the milestone of 200,000 deaths from COVID-19."                                                                                                                                                                                                                                                                                                  
## [17] "This number represented 20% of worldwide fatalities, despite the US having only 4% of the world's population."                                                                                                                                                                                                                                                                                
## [18] "Two weeks prior to the debate, Supreme Court Justice Ruth Bader Ginsburg died from cancer."                                                                                                                                                                                                                                                                                                   
## [19] "Ginsburg was one of four Supreme Court justices who are commonly considered liberal; the other five justices are commonly considered to be conservative."                                                                                                                                                                                                                                     
## [20] "The day after Ginsberg's funeral, Trump nominated conservative Amy Coney Barrett."                                                                                                                                                                                                                                                                                                            
## [21] "Senate Republicans, under the leadership of Senate Majority Leader Mitch McConnell, moved swiftly, promising to vote on her nomination before Election Day."                                                                                                                                                                                                                                  
## [22] "The move was controversial, since the same Senate Republicans had refused to consider a Supreme Court nomination by President Barack Obama in an election year."                                                                                                                                                                                                                              
## [23] "Format and debate\nWill you shut up, man? redirects here."                                                                                                                                                                                                                                                                                                                                    
## [24] "For Why don't you shut up?, the phrase uttered by King Juan Carlos I of Spain to Hugo Chávez, see ¿Por qué no te callas?"                                                                                                                                                                                                                                                                     
## [25] "The debate was divided into six segments: Trump's and Biden's records, the Supreme Court, the COVID-19 pandemic, race and violence in cities, election integrity, and the economy."                                                                                                                                                                                                           
## [26] "Each was approximately 15 minutes in length; Wallace introduced each topic and gave each candidate two minutes to speak, followed by facilitated discussion between them."                                                                                                                                                                                                                    
## [27] "The allotted time was generally not upheld; Trump repeatedly interrupted and criticized Biden during Biden's answers to the initial questions as well as during the facilitated discussions, and was chastised by Wallace several times for doing so."                                                                                                                                        
## [28] "On several occasions, Wallace pleaded with Trump to respect the rules and norms of the debate."                                                                                                                                                                                                                                                                                               
## [29] "At one point, Biden refused to answer a question given by Wallace, leading to Trump interrupting him."                                                                                                                                                                                                                                                                                        
## [30] "Biden then remarked to Trump, Will you shut up, man?"                                                                                                                                                                                                                                                                                                                                         
## [31] "Additionally, Biden called Trump a clown during the discussion about healthcare plans."                                                                                                                                                                                                                                                                                                       
## [32] "At one point during the debate, Biden and Wallace pressed Trump to condemn white supremacy groups."                                                                                                                                                                                                                                                                                           
## [33] "When Trump replied Give me a name..., Biden responded with The Proud Boys."                                                                                                                                                                                                                                                                                                                   
## [34] "Trump then said Proud Boys, stand back and stand by, a remark interpreted by some members of that far-right group, as well as others, as a call to arms.When asked about his position on police reform, Biden called for an increase in police funding, in opposition to left-wing rhetoric calling for a defunding of police."                                                               
## [35] "He explained such funds would be used to hire psychologists or psychiatrists who would accompany police officers during 9-1-1 calls in order to defuse potentially violent situations and reduce the use of force, and improve officer training."                                                                                                                                             
## [36] "Fact checkers challenged many of Trump's statements."                                                                                                                                                                                                                                                                                                                                         
## [37] "Trump falsely said that he brought back (college) football; as he had commented on his wish for the conferences to play, but took no official action."                                                                                                                                                                                                                                        
## [38] "Trump also repeated the claim that he ''got back'' Seattle and Minneapolis from left-wing protesters, and continued to repeat conspiracy theories about voter fraud."                                                                                                                                                                                                                         
## [39] "He said, without evidence, that drug prices will fall 80 or 90 percent, in reference to his efforts to cut drug prices and exaggerated that he is making insulin at prices so cheap, it's like water, despite insulin prices remaining fixed at about $300 per vial."                                                                                                                         
## [40] "Trump also misleadingly said that the U.S. economy before the pandemic was the greatest economy in the history of our country; although GDP growth was high in the first three years of the Trump Presidency, it was higher under Presidents Dwight D. Eisenhower, Lyndon B. Johnson, and Bill Clinton, and the unemployment rate was lower under Eisenhower."                                
## [41] "Nominal GDP was higher than at any point in US history, but this is generally expected to be the case."                                                                                                                                                                                                                                                                                       
## [42] "When Biden brought up Trump's March 2020 remarks about injecting disinfectant to treat COVID-19, Trump claimed that they had been made sarcastically."                                                                                                                                                                                                                                        
## [43] "Trump then stated that he brought back 700,000 manufacturing jobs; a false figure given that the actual number was 487,000."                                                                                                                                                                                                                                                                  
## [44] "Biden then made several false claims, claiming that under Trump, the trade deficit with China grew and violent crime went up (only the national murder rate increased)."                                                                                                                                                                                                                      
## [45] "Trump criticized Biden's handling of the 2009 swine flu pandemic, a pandemic in which an estimated 60 million cases in the United States occurred, with an estimated death toll of about 12,000."                                                                                                                                                                                             
## [46] "When Biden mentioned that Trump should get a lot smarter, Trump said, Don't ever use the word smart with me, don't ever use that word [...] Because there's nothing smart about you, Joe, and incorrectly stated that Biden forgot where he went to college, referring to a video in which Biden talks about announcing his first Senate campaign on the campus of Delaware State University."
## [47] "Reception and aftermath\nA post-debate CNN/SSRS poll found that 60% of debate-viewers thought that Biden had won and 28% thought Trump had, with a margin of error of six points."                                                                                                                                                                                                            
## [48] "According to a CBS News poll taken following the debate, 48% of people thought Biden won, 41% of people thought Trump won, while 10% considered it a tie, with a margin of error of three points."                                                                                                                                                                                            
## [49] "In the same poll, 83% of the respondents believed the tone of the debate was negative, while 17% believed it was positive."                                                                                                                                                                                                                                                                   
## [50] "The debate was largely seen negatively across the political spectrum."                                                                                                                                                                                                                                                                                                                        
## [51] "The debate was widely criticized by commentators and journalists."                                                                                                                                                                                                                                                                                                                            
## [52] "It was called a hot mess, inside a dumpster fire, inside a train wreck and a disgrace (CNN's Jake Tapper); a shitshow (CNN's Dana Bash); mud-wrestling (ABC's Martha Raddatz); the worst presidential debate I have ever seen in my life (ABC's George Stephanopoulos); and the single worst debate I have ever covered in my two decades of doing this job (CNN's Chris Cillizza)."          
## [53] "The New York Times editorial board called the debate excruciating and wrote: After five years of conditioning, the president's ceaseless lies, insults and abuse were no less breath-taking to behold."                                                                                                                                                                                       
## [54] "The Washington Post editorial board called the debate a disgrace and demonstrated that Trump's assault on democracy is escalating."                                                                                                                                                                                                                                                           
## [55] "ABC White House correspondent Jonathan Karl said that Trump came across as a bully in the debate."                                                                                                                                                                                                                                                                                            
## [56] "According to the Washington Examiner, some conservatives criticized Wallace for an alleged bias against Trump due to Wallace's frequent interruptions of Trump."                                                                                                                                                                                                                              
## [57] "After moderating the debate, Wallace described his performance as moderator as a terrible missed opportunity and remarked that he had not been prepared for Trump's behaviour."                                                                                                                                                                                                               
## [58] "In response to the failure of the debate and subsequent criticism, the Commission on Presidential Debates indicated that it would modify future debates to encourage a more civilized and orderly discussion."                                                                                                                                                                                
## [59] "While Biden said that he was open to changes, Trump rejected the idea, suggesting that changes would erode his advantage."                                                                                                                                                                                                                                                                    
## [60] "Despite criticism of his moderation, the CPD defended Wallace's moderation ability, commending his professionalism and skill."                                                                                                                                                                                                                                                                
## [61] "Trump's stand by remarks received criticism.Rick Santorum, a former Republican senator, later said that it was a huge mistake by Trump not to condemn white supremacy properly during the debate.Fox & Friends co-host Brian Kilmeade criticized Trump for not condemning white supremacy, saying that Trump ruined the biggest layup in the history of debates by not doing so."             
## [62] "Trump's team disagreed with these criticisms, arguing that Trump has continuously denounced white supremacists and did so twice during the debate."                                                                                                                                                                                                                                           
## [63] "The day after the debate, Trump said, I don't know who Proud Boys are, but whoever they are, they have to stand down.On October 1, Trump said on Sean Hannity's show: I've said it many times, and let me be clear again: I condemn the KKK."                                                                                                                                                 
## [64] "I condemn all white supremacists."                                                                                                                                                                                                                                                                                                                                                            
## [65] "I condemn the Proud Boys."                                                                                                                                                                                                                                                                                                                                                                    
## [66] "I don't know much about the Proud Boys, almost nothing."                                                                                                                                                                                                                                                                                                                                      
## [67] "But I condemn that."                                                                                                                                                                                                                                                                                                                                                                          
## [68] "Researcher Rita Katz, executive director of SITE Intelligence Group, told The Washington Post that Proud Boys memberships on Telegram channels grew nearly 10 percent after the debate."                                                                                                                                                                                                      
## [69] "Proud Boys merchandise featuring the phrases stand back and stand by appeared online after the debate and was subsequently banned from sites including Amazon Marketplace and Teespring; it remained available on eBay as of October 1."                                                                                                                                                      
## [70] "The debate had a total of at least 73.1 million viewers on television, according to Nielsen ratings."                                                                                                                                                                                                                                                                                         
## [71] "It was the third most watched debate in U.S. history, behind the first debate between Trump and Hillary Clinton in 2016 (84 million), and the only debate between Jimmy Carter and Ronald Reagan in 1980 (80.6 million)."                                                                                                                                                                     
## [72] "The television viewership declined 13% compared to the debate for the first presidential debate of 2016, but an unknown number of people watched or listened to the debate via live-streaming or radio, so the total audience likely surpassed the 2016 record.\""